package com.g2room.driving.grap.news;

import java.io.IOException;
import java.net.MalformedURLException;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.eclipse.jetty.util.StringUtil;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.alibaba.fastjson.JSON;
import com.g2room.driving.grap.GenericGrap;
import com.g2room.driving.grap.news.model.po.NewsInfo;
import com.g2room.myuall.grap.ky.model.po.GrapErr;
import com.g2room.myuall.util.DateUtil;
import com.g2room.myuall.util.DbUtil;
import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;

public class GrapCqqqData {
	private static String baseUrl = "http://cq.qq.com";
	private String soruce = "大渝网";
	private static Map<String, String> urlMap = new HashMap<String, String>();
	static {
		urlMap.put("http://cq.qq.com/CQxinwen/chongqi/cqnews.htm","时政民生");
		urlMap.put("http://cq.qq.com/CQxinwen/society/socnews.htm","社会新闻");
	}
	
	public static void main(String[] args) throws Exception {
		GrapCqqqData gcd = new GrapCqqqData();
		for(String url : urlMap.keySet()) {
			gcd.getList(url, urlMap.get(url));
		}
	}

	/**
	 * 获取头部导航菜单
	 * 
	 * @param url
	 * @throws FailingHttpStatusCodeException
	 * @throws MalformedURLException
	 * @throws IOException
	 */
	public void getList(String url, String type) {
		Document doc = null;
		try {
			doc = GenericGrap.getDocByIp(url);
		} catch (IOException e) {
			GrapErr ge = new GrapErr();
			ge.setUrl(url);
			ge.setType("1");
			ge.setContent("获取链接错误");
			ge.setDescription("获取Type错误：" + e.getMessage());
			insert2ErrDb(ge);
			return;
		}

		if (null == doc) return;
		Element el = null;
		List<String> urls = new ArrayList<String>();
		try {
			el = doc.getElementById("PageSet");
			/**
			 * <li style="list-style-type: none;">
<span class="ll">·<a target="_blank" class="black" href="/a/20161117/026748.htm">好消息！在北环可以坐长途汽车了</a></span><span class="lr">（11月17日14:24)</span>
</li>
			 */
			Elements aels = el.getElementsByTag("a");
			Elements ptels = el.getElementsByAttributeValue("class", "lr");
			if (aels == null || aels.size() == 0) return;
			Date pt = getMaxPublishTime();
			Date cruPt = null;
			for (int i = 0; i < aels.size(); i++) {
				try {
					// 判断是否已经抓取过
					el = ptels.get(i);
					if(null != el && el.text().length() > 5) {
						String time = el.text().replace("（", "") .replace(")", "").replace("(", "") .replace("）", "");
						cruPt = DateUtil.string2Date(Calendar.getInstance().get(Calendar.YEAR) + "年" + time, "yyyy年MM月dd日HH:mm");
						if(cruPt.compareTo(pt) > 0) {
							urls.add(baseUrl + aels.get(i).attr("href"));
						}
					}
				} catch (Exception e) {
					e.printStackTrace();
				}
			}
		} catch (Exception e) {
			GrapErr ge = new GrapErr();
			ge.setUrl(url);
			ge.setType("1");
			ge.setContent("获取内容错误");
			ge.setDescription("获取Type错误：" + e.getMessage());
			insert2ErrDb(ge);
			return;
		}
		System.out.println(type + "本次抓取数据：" + urls.size());

		for (String durl : urls) {
			getDetail(durl, type);
		}
	}

	public void getDetail(String url, String typeName) {
		Document doc = null;
		try {
			doc = GenericGrap.getDocByIp(url);
		} catch (Exception e) {
			GrapErr ge = new GrapErr();
			ge.setUrl(url);
			ge.setType("2");
			ge.setContent("");
			ge.setDescription("获取Detail 链接错误：" + e.getMessage());
			insert2ErrDb(ge);
			return;
		}

		Element el = null;
		try {
			el = doc.getElementById("C-Main-Article-QQ");
			
			Elements els = el.getElementsByTag("div");
			if(null == els || els.isEmpty()) return;
			
			NewsInfo ni = new NewsInfo();
			Element headEl = els.get(0);
			
			Elements tempEls = headEl.getElementsByTag("h1");
			if(null != tempEls && !tempEls.isEmpty()) ni.setTitle(tempEls.first().text());
			
			tempEls = headEl.getElementsByAttributeValue("class", "color-a-1");
			if(null != tempEls && !tempEls.isEmpty()) ni.setSource(tempEls.first().text());
			
			tempEls = headEl.getElementsByAttributeValue("class", "color-a-3");
			if(null != tempEls && !tempEls.isEmpty()) ni.setAuthor(tempEls.first().text());
			
			tempEls = headEl.getElementsByAttributeValue("class", "article-time");
			if(null != tempEls && !tempEls.isEmpty()) {
				String pt = tempEls.first().text();
				if(StringUtil.isNotBlank(pt)) {
					ni.setPublishTimeStr(pt + ":00");
				}
			}
			
			el = doc.getElementById("Cnt-Main-Article-QQ");
			if(null == el) return;
			
			tempEls = el.children();
			boolean flag = false;
			for(Element e : tempEls) {
				if(!flag && e.text().contains("视频仅供延伸阅读")) flag = true;
				if(flag) e.remove();
			}
			ni.setContent(el.html());
			ni.setTypeName(typeName);
			ni.setSourceUrl(url);
			insert2ComDb(ni);
		} catch (Exception e) {
			GrapErr ge = new GrapErr();
			ge.setUrl(url);
			ge.setType("3");
			ge.setContent("");
			ge.setDescription("获取Detail 内容错误：" + e.getMessage());
			insert2ErrDb(ge);
			return;
		}
	}

	public void insert2ComDb(NewsInfo ni) {
		String sql = "insert into news_info(author,title,summary,content,publish_time,source,source_url,type_name) values(?,?,?,?,?,?,?,?)";
		PreparedStatement ps = null;
		try {
			ps = DbUtil.getNewsConn().prepareStatement(sql);
			/**
author,title,summary,content,publish_time,source,source_url,type_name

			 */
			ps.setString(1, ni.getAuthor());
			ps.setString(2, ni.getTitle());
			ps.setString(3, ni.getSummary());
			ps.setString(4, ni.getContent());
			ps.setString(5, ni.getPublishTimeStr());

			ps.setString(6, ni.getSource());
			ps.setString(7, ni.getSourceUrl());
			ps.setString(8, ni.getTypeName());
			ps.execute();
		} catch (Exception e) {
			GrapErr ge = new GrapErr();
			ge.setUrl(ni.getSourceUrl());
			ge.setType("5");
			ge.setContent(JSON.toJSONString(ni));
			ge.setDescription("获取Detail错误：" + e.getMessage());
			insert2ErrDb(ge);
			return;
		} finally {
			if (null != ps) {
				try {
					ps.close();
				} catch (SQLException e) {

				}
			}
		}
	}
	
	public static Date getMaxPublishTime() {
		Date pt = null;
		try {
			Connection connection = DbUtil.getNewsConn();
			Statement statement = connection.createStatement();
			String sql = "select max(publish_time) as pt from news_info";
			ResultSet rs = statement.executeQuery(sql);
			while(rs.next()) {
				pt = rs.getTimestamp("pt");
			}
			rs.close();
			statement.close();
		} catch(Exception e) {
			e.printStackTrace();
		}
		return pt;
	}
	
	public void insert2ErrDb(GrapErr ge) {
		PreparedStatement grapPs = null;
		try {
			grapPs = DbUtil.getNewsConn().prepareStatement("insert into grap_err(description,url,content,type) values(?,?,?,?)");
			grapPs.setString(1, ge.getDescription());
			grapPs.setString(2, ge.getUrl());
			grapPs.setString(3, ge.getContent());
			grapPs.setString(4, ge.getType());
			grapPs.execute();
			grapPs.close();
		} catch (Exception e) {

		} finally {
			if (null != grapPs) {
				try {
					grapPs.close();
				} catch (SQLException e) {

				}
			}
		}
	}
}
